from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
import pymysql
import re
import urllib.request
import datetime
import os
import time
browser=webdriver.Chrome()

# 1.连接数据库，查询网址
# 2.打开浏览器采集数据
# 3.关闭选项卡,数据入库
# 4.进行下一波循环
# 5.关闭数据库,关闭浏览器

db=pymysql.connect(host="localhost", user='root', password='root', db='spiders')
cursor=db.cursor()
sql = 'select id,title,link from cmf_jrtt_list'
cursor.execute(sql)
results=cursor.fetchall()
for row in results:
    print(row)
    time.sleep(3)
    browser.get(row[2])
    item = {}
    try:
        #####################文章类begin#########################
        title1 = browser.find_element_by_xpath("/html/body/div/div[2]/div[2]/div[1]/h1").text
        writer1 = browser.find_element_by_xpath("/html/body/div/div[2]/div[2]/div[1]/div[1]/span[1]").text
        publishtime1 = browser.find_element_by_xpath("/html/body/div/div[2]/div[2]/div[1]/div[1]/span[2]").text
        content1 = browser.find_element_by_xpath("/html/body/div/div[2]/div[2]/div[1]/div[2]").get_attribute(
            'innerHTML')
        #####################文章类end#########################

        #####################问答类begin#########################
        title2 = browser.find_element_by_xpath("/html/body/div/div[2]/div[2]/div[1]/div[2]/h1").text
        content2 = browser.find_element_by_xpath(
            "/html/body/div/div[2]/div[2]/div[1]/div[5]/div[1]/div/div[1]/div[3]/div[1]").get_attribute('innerHTML')
        writer2 = browser.find_element_by_xpath(
            "/html/body/div/div[2]/div[2]/div[1]/div[5]/div[1]/div/div[1]/div[2]/a[2]").text
        publishtime2 = browser.find_element_by_xpath(
            "/html/body/div/div[2]/div[2]/div[1]/div[5]/div[1]/div/div[1]/div[2]/a[3]").text
        #####################问答类end#########################
    except:
        # print('抓不到数据')
        pass
    if title1 != "":
        item['title'] = title1
    else:
        item['title'] = title2

    if writer1 != "":
        item['writer'] = writer1
    else:
        item['writer'] = writer2
    if publishtime1 != "":
        item['publishtime'] = publishtime1
    else:
        item['publishtime'] = publishtime2
    if content1 != "":
        item['content'] = content1
    else:
        item['content'] = content2
    listurl = re.findall(r'src="(http:\/\/p.*?)"', item['content'])
    # print(listurl)
    for imgurl in listurl:
        req = urllib.request.urlopen(imgurl)
        buf = req.read()
        dirname='picture/'+str(datetime.date.today())
        # print(dirname)
        if not os.path.exists(dirname):
            os.mkdir(dirname)
        filepath = dirname + '/' + str(int(time.time()*1000)) + '.jpg'
        with open(filepath, 'wb+') as fp:
            fp.write(buf)
        fp.close()
        item['content']=item['content'].replace(imgurl,r"/"+filepath)
    print(item)
    db = pymysql.connect(host="localhost", user='root', password='root', db='spiders')
    cursor = db.cursor()
    sql='insert into article(`title`,`writer`,`publish`,`content`,`sourceurl`,`createtime`) values (%s,%s,%s,%s,%s,%s)'
    try:
        cursor.execute(sql,(item['title'],item['writer'],item['publishtime'],pymysql.escape_string(item['content']),row[2],datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
        db.commit()
        print('插入成功')
    except:
        db.rollback()
        print('插入失败')

browser.quit()
db.close()


